{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YONnGjpAYUdU"
      },
      "source": [
        "\n",
        "\u003ca href=\"https://colab.research.google.com/github/google-research/bigbird/blob/master/bigbird/summarization/pubmed.ipynb\" target=\"_parent\"\u003e\u003cimg src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/\u003e\u003c/a\u003e"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zrtR2urJV3ST"
      },
      "source": [
        "##### Copyright 2020 The BigBird Authors\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xyasTfa-LVLe"
      },
      "outputs": [],
      "source": [
        "# Copyright 2020 The BigBird Authors. All Rights Reserved.\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License.\n",
        "# =============================================================================="
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fcZZRDx505hq"
      },
      "source": [
        "## Set Up"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N94UyOdA0mCO"
      },
      "outputs": [],
      "source": [
        "!pip install git+https://github.com/google-research/bigbird.git -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0irPwcbBYvDV"
      },
      "outputs": [],
      "source": [
        "from bigbird.core import flags\n",
        "from bigbird.core import modeling\n",
        "from bigbird.core import utils\n",
        "from bigbird.summarization import run_summarization\n",
        "import tensorflow.compat.v2 as tf\n",
        "import tensorflow_datasets as tfds\n",
        "import tensorflow_text as tft\n",
        "from tqdm import tqdm\n",
        "import sys\n",
        "\n",
        "FLAGS = flags.FLAGS\n",
        "if not hasattr(FLAGS, \"f\"): flags.DEFINE_string(\"f\", \"\", \"\")\n",
        "FLAGS(sys.argv)\n",
        "\n",
        "tf.enable_v2_behavior()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AJexg2zsxfHo"
      },
      "source": [
        "## Set options"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rph2sJ75kBNA"
      },
      "outputs": [],
      "source": [
        "FLAGS.data_dir = \"tfds://scientific_papers/pubmed\"\n",
        "FLAGS.attention_type = \"block_sparse\"\n",
        "FLAGS.couple_encoder_decoder = True\n",
        "FLAGS.max_encoder_length = 2048  # on free colab only lower memory GPU like T4 is available\n",
        "FLAGS.max_decoder_length = 256\n",
        "FLAGS.block_size = 64\n",
        "FLAGS.learning_rate = 1e-5\n",
        "FLAGS.num_train_steps = 10000\n",
        "FLAGS.attention_probs_dropout_prob = 0.0\n",
        "FLAGS.hidden_dropout_prob = 0.0\n",
        "FLAGS.vocab_model_file = \"gpt2\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zuxI3V_3j57Y"
      },
      "outputs": [],
      "source": [
        "transformer_config = flags.as_dictionary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kRF4TUEQxjXJ"
      },
      "source": [
        "## Define summarization model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5RX4ow05URf8"
      },
      "outputs": [],
      "source": [
        "from tensorflow.python.ops.variable_scope import EagerVariableStore\n",
        "container = EagerVariableStore()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J3yNdo5toQwq"
      },
      "outputs": [],
      "source": [
        "with container.as_default():\n",
        "  model = modeling.TransformerModel(transformer_config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DXOY78vbqHX9"
      },
      "outputs": [],
      "source": [
        "@tf.function(experimental_compile=True)\n",
        "def fwd_bwd(features, labels):\n",
        "  with tf.GradientTape() as g:\n",
        "    (llh, logits, pred_ids), _ = model(features, target_ids=labels,\n",
        "                                       training=True)\n",
        "    loss = run_summarization.padded_cross_entropy_loss(\n",
        "        logits, labels,\n",
        "        transformer_config[\"label_smoothing\"],\n",
        "        transformer_config[\"vocab_size\"])\n",
        "  grads = g.gradient(loss, model.trainable_weights)\n",
        "  return loss, llh, logits, pred_ids, grads"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DzoTMyQlxsRo"
      },
      "source": [
        "## Dataset pipeline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Oo-NQTDZZs51"
      },
      "outputs": [],
      "source": [
        "train_input_fn = run_summarization.input_fn_builder(\n",
        "        data_dir=FLAGS.data_dir,\n",
        "        vocab_model_file=FLAGS.vocab_model_file,\n",
        "        max_encoder_length=FLAGS.max_encoder_length,\n",
        "        max_decoder_length=FLAGS.max_decoder_length,\n",
        "        substitute_newline=FLAGS.substitute_newline,\n",
        "        is_training=True)\n",
        "dataset = train_input_fn({'batch_size': 2})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hRvmfaNUi-V5"
      },
      "outputs": [],
      "source": [
        "# inspect at a few examples\n",
        "for ex in dataset.take(3):\n",
        "  print(ex)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lYCyGH56zOOU"
      },
      "source": [
        "## Check outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5uQOwyGQzRKt"
      },
      "outputs": [],
      "source": [
        "with container.as_default():\n",
        "  loss, llh, logits, pred_ids, grads = fwd_bwd(ex[0], ex[1])\n",
        "print('Loss: ', loss)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qz_LdCCdyDCR"
      },
      "source": [
        "## (Optionally) Load pretrained model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rRa2dD1RzLN4"
      },
      "outputs": [],
      "source": [
        "# For training from scratch use\n",
        "# ckpt_path = 'gs://bigbird-transformer/pretrain/bigbr_base/model.ckpt-0'\n",
        "# For quick check continue from trained checkpoint\n",
        "ckpt_path = 'gs://bigbird-transformer/summarization/pubmed/roberta/model.ckpt-300000'\n",
        "ckpt_reader = tf.compat.v1.train.NewCheckpointReader(ckpt_path)\n",
        "loaded_weights = []\n",
        "for v in tqdm(model.trainable_weights, position=0):\n",
        "  try:\n",
        "    val = ckpt_reader.get_tensor(v.name[:-2])\n",
        "  except:\n",
        "    val = v.numpy()\n",
        "  loaded_weights.append(val)\n",
        "\n",
        "model.set_weights(loaded_weights)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r6-BziYxzL3U"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IWjkDvu9k7ie"
      },
      "outputs": [],
      "source": [
        "opt = tf.keras.optimizers.Adam(FLAGS.learning_rate)\n",
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "\n",
        "for i, ex in enumerate(tqdm(dataset.take(FLAGS.num_train_steps), position=0)):\n",
        "  loss, llh, logits, pred_ids, grads = fwd_bwd(ex[0], ex[1])\n",
        "  opt.apply_gradients(zip(grads, model.trainable_weights))\n",
        "  train_loss(loss)\n",
        "  if i% 10 == 0:\n",
        "    print('Loss = {} '.format(train_loss.result().numpy()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mXjkdtyKMAbP"
      },
      "source": [
        "## Eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cc3OatxlMCk3"
      },
      "outputs": [],
      "source": [
        "@tf.function(experimental_compile=True)\n",
        "def fwd_only(features, labels):\n",
        "  (llh, logits, pred_ids), _ = model(features, target_ids=labels,\n",
        "                                       training=False)\n",
        "  return llh, logits, pred_ids"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-Mq_xhMzef42"
      },
      "outputs": [],
      "source": [
        "eval_input_fn = run_summarization.input_fn_builder(\n",
        "        data_dir=FLAGS.data_dir,\n",
        "        vocab_model_file=FLAGS.vocab_model_file,\n",
        "        max_encoder_length=FLAGS.max_encoder_length,\n",
        "        max_decoder_length=FLAGS.max_decoder_length,\n",
        "        substitute_newline=FLAGS.substitute_newline,\n",
        "        is_training=False)\n",
        "eval_dataset = eval_input_fn({'batch_size': 2})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rqPN4R8kerUG"
      },
      "outputs": [],
      "source": [
        "eval_llh = tf.keras.metrics.Mean(name='eval_llh')\n",
        "\n",
        "for ex in tqdm(eval_dataset, position=0):\n",
        "  llh, logits, pred_ids = fwd_only(ex[0], ex[1])\n",
        "  eval_llh(llh)\n",
        "print('Log Likelihood = {}'.format(eval_llh.result().numpy()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SA-MksK2DEOn"
      },
      "source": [
        "### Print predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a-FD37ZLxr8R"
      },
      "outputs": [],
      "source": [
        "tokenizer = tft.SentencepieceTokenizer(\n",
        "        model=tf.io.gfile.GFile(FLAGS.vocab_model_file, \"rb\").read())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8RXHSgpKFG1F"
      },
      "outputs": [],
      "source": [
        "_, _, pred_ids = fwd_only(ex[0], ex[1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 365,
          "status": "ok",
          "timestamp": 1607929344388,
          "user": {
            "displayName": "Manzil Zaheer",
            "photoUrl": "",
            "userId": "06259716656099187509"
          },
          "user_tz": 480
        },
        "id": "BvEFgoXJxQOa",
        "outputId": "72d452bd-c64c-46a2-a1ee-2aa398f6f2cd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Article:\n",
            " [b'although injuries to the flexor tendons in the zone ii region look trivial , sustained commitment of the patient , the surgeon and the therapist is necessary to get a reasonable functional outcome . as our institute \\xe2\\x81\\x87 is situated in an industrial corridor of the city , most of our patients are manual workers with poor compliance . conforming to the established practice , we tried various early mobilization protocols after zone ii flexor tendon repair . \\xe2\\x81\\x87 as these protocols demanded a level of understanding and a degree of dedication from the patients , our results were suboptimal , with a high incidence of proximal interphalangeal joint ( pip ) joint flexion contractures and tendon ruptures . \\xe2\\x81\\x87 hence , in our institute immobilization became the norm for patients who were not expected to be compliant . \\xe2\\x81\\x87 to improve the results in such patients , we thought of implementing a new rehabilitation protocol that could entirely be under the control of the therapist , without the active participation of the patient . \\xe2\\x81\\x87 ultrasound therapy is being used safely after tendon repair , during the remodeling phase ( after three weeks ) as an adjunct to mobilization to improve the tendon gliding . \\xe2\\x81\\x87 we thought of using this mode of treatment during the earlier phase of tendon healing . \\xe2\\x81\\x87 such findings formed the basis , for the use of ultrasound to accelerate tendon healing and to prevent adhesions . \\xe2\\x81\\x87 though many clinical trials on animals are reported in the literature , there is no clinical study in the humans regarding the use of ultrasound during the early healing phase of the repaired tendons . \\xe2\\x81\\x87 encouraged by the in vitro and in vivo animal studies , we conducted the trial by selecting the patients at random and administered pulsed ultrasound therapy of different frequencies and intensities . \\xe2\\x81\\x87 the results of the ultrasound therapy in the three groups were analyzed and compared with each other and also with the immobilization group . \\xe2\\x81\\x87 this is a prospective study , done over a period of 5 years from january 2008 to january 2013 , involving a total of 100 patients and 139 digits with zone ii flexor tendon injuries . \\xe2\\x81\\x87 patients with isolated injury to the fds or fdp tendon , multiple level injuries of the flexor tendons , associated injury to the extensor apparatus and fractures were excluded . \\xe2\\x81\\x87 all the cases were operated as emergency procedures by senior residents and hence , for tendon repair technically less demanding two - strand modified kessler mason suture was used . after surgery , patients were allocated to ultrasound therapy , as demonstrated in the consort guideline flowchart [ figure 1 ] . \\xe2\\x81\\x87 the patients were asked to draw a card indiscriminately from an envelope containing a pack of cards labeled as ultrasound or immobilization in 2:1 ratio . \\xe2\\x81\\x87 thus , the treatment groups were randomized and the therapists had no control over the selection of the patients . \\xe2\\x81\\x87 after surgery , hands were immobilized for 3 weeks using dorsal slab with wrist kept in neutral position , mp joints in 70 degrees flexion and ip joints in extension . \\xe2\\x81\\x87 after 3 weeks , pop slab was removed and the patients were advised to attend therapy daily . from 3 to 6 weeks the therapy consisted of scar massage , active mobilization exercises , blocking exercises and place hold techniques . after 6 weeks , passive stretching and resisted exercises were added . \\xe2\\x81\\x87 after 8 weeks , the patients were allowed to lift weights and to join work . by this protocol \\xe2\\x81\\x87 after surgery hands were immobilized using dorsal slab with wrist in neutral , mp joints in 70 degrees flexion and ip joints in extension . with the dorsal splint in place , the dressings on the volar aspect were removed . \\xe2\\x81\\x87 the ultrasonic treatment head was placed over the site of tendon repair and gently moved in order to iron out the irregularities in the near field and to avoid standing waves due to reflection . \\xe2\\x81\\x87 the ultrasonic head was applied gently and moved continuously over the region of tendon repair to begin with , since january 2008 , ultrasound of 1-mhz frequency at an intensity of 0.7 w / cm was administered from the seventh postoperative day . \\xe2\\x81\\x87 twenty - six patients with involvement of a total of thirty eight digits were treated . \\xe2\\x81\\x87 twelve patients had associated digital nerve involvement . with the idea of further refining the technique , since august 2010 \\xe2\\x81\\x87 , we started ultrasound therapy of 1-mhz frequency at an intensity of 0.3 w / cm from the third postoperative day . \\xe2\\x81\\x87 we reduced the intensity due to fear of impaired skin healing as we had started the therapy earlier than the first group . \\xe2\\x81\\x87 as there was significant reduction in the percentage of excellent results in group 2 , from october 2011 , ultrasound therapy of 3 mhz frequency at an intensity of 0.5 w / cm was administered from the fifth postoperative day . due to cases of wound dehiscence \\xe2\\x81\\x87 this protocol was started for 27 patients with involvement of a total of 33 fingers . \\xe2\\x81\\x87 three patients developed wound dehiscence within 2 days of starting ultrasound therapy and were dropped from the study . out of 24 patients , 21 were males and 3 were females . \\xe2\\x81\\x87 the splint was removed after 3 weeks and mobilization programe was commenced in addition to the ultrasound therapy . \\xe2\\x81\\x87 active ranges of movements at pip and dip joints were measured after 3 weeks and at weekly intervals . \\xe2\\x81\\x87 after surgery , hands were immobilized for 3 weeks using dorsal slab with wrist kept in neutral position , mp joints in 70 degrees flexion and ip joints in extension . \\xe2\\x81\\x87 after 3 weeks , pop slab was removed and the patients were advised to attend therapy daily . from 3 to 6 weeks the therapy consisted of scar massage , active mobilization exercises , blocking exercises and place hold techniques . after 6 weeks , passive stretching and resisted exercises were added . after 8 weeks \\xe2\\x81\\x87 , the patients were allowed to lift weights and to join work . by this protocol \\xe2\\x81\\x87 after surgery hands were immobilized using dorsal slab with wrist in neutral , mp joints in 70 degrees flexion and ip joints in extension . with the dorsal splint in place , the dressings on the volar aspect were removed . \\xe2\\x81\\x87 the ultrasonic treatment head was placed over the site of tendon repair and gently moved in order to iron out the irregularities in the near field and to avoid standing waves due to reflection . \\xe2\\x81\\x87 the ultrasonic head was applied gently and moved continuously over the region of tendon repair to begin with , since january 2008 , ultrasound of 1-mhz frequency at an intensity of 0.7 w / cm was administered from the seventh postoperative day . \\xe2\\x81\\x87 twenty - six patients with involvement of a total of thirty eight digits were treated . \\xe2\\x81\\x87 twelve patients had associated digital nerve involvement . with the idea of further refining the technique , since august 2010 \\xe2\\x81\\x87 , we started ultrasound therapy of 1-mhz frequency at an intensity of 0.3 w / cm from the third postoperative day . \\xe2\\x81\\x87 we reduced the intensity due to fear of impaired skin healing as we had started the therapy earlier than the first group . \\xe2\\x81\\x87 one patient developed wound dehiscence and was dropped from the study . out of 18 patients , 15 were males and 3 were females . \\xe2\\x81\\x87 six patients had digital nerve involvement . as there was significant reduction in the percentage of excellent results in group 2 , from october 2011 , ultrasound therapy of 3 mhz frequency at an intensity of 0.5 w / cm \\xe2\\x81\\x87 this protocol was started for 27 patients with involvement of a total of 33 fingers . \\xe2\\x81\\x87 three patients developed wound dehiscence within 2 days of starting ultrasound therapy and were dropped from the study . out of 24 patients , 21 were males and 3 were females . \\xe2\\x81\\x87 in all the ultrasound therapy groups , the splint was removed after 3 weeks and mobilization programe was commenced in addition to the ultrasound therapy . \\xe2\\x81\\x87 to begin with , since january 2008 , ultrasound of 1-mhz frequency at an intensity of 0.7 w / cm was administered from the seventh postoperative day . \\xe2\\x81\\x87 twenty - six patients with involvement of a total of thirty eight digits were treated . \\xe2\\x81\\x87 with the idea of further refining the technique , since august 2010 , we started ultrasound therapy of 1-mhz frequency at an intensity of 0.3 w / cm from the third postoperative day . \\xe2\\x81\\x87 we reduced the intensity due to fear of impaired skin healing as we had started the therapy earlier than the first group . \\xe2\\x81\\x87 one patient developed wound dehiscence and was dropped from the study . out of 18 patients , 15 were males and 3 were females . \\xe2\\x81\\x87 as there was significant reduction in the percentage of excellent results in group 2 , from october 2011 , ultrasound therapy of 3 mhz frequency at an intensity of 0.5 w / cm was administered from the fifth postoperative day . due to cases of wound dehiscence \\xe2\\x81\\x87 this protocol was started for 27 patients with involvement of a total of 33 fingers . \\xe2\\x81\\x87 three patients developed wound dehiscence within 2 days of starting ultrasound therapy and were dropped from the study . out of 24 patients , 21 were males and 3 were females . \\xe2\\x81\\x87 the splint was removed after 3 weeks and mobilization programe was commenced in addition to the ultrasound therapy . \\xe2\\x81\\x87 active ranges of movements at pip and dip joints were measured after 3 weeks and at weekly intervals . \\xe2\\x81\\x87 the patients who were dropped from the study due to wound complications in the early phase were not included in the analysis of the results . \\xe2\\x81\\x87 comparison of results between various protocols by the end of 3 months is given in tables 1 and 2 . \\xe2\\x81\\x87 comparison of results between various protocols at 3 months ( original strickland ) comparison of excellent and good results in various protocols ( original strickland ) in ultrasound therapy group 1 protocol , out of 36 digits , 16 digits achieved pip + dip joint flexion range between 150 and 175 and 10 digits between 125 and 150 in 8 weeks time . \\xe2\\x81\\x87 the range in the remaining digits was between 90 and 120. extensor lag was nil in 34 digits by the end of 6 weeks . by the end of 8 weeks , none of the digits had extensor lag . \\xe2\\x81\\x87 grip strength of 95% compared to normal hand was achieved by 16 digits and 10 digits achieved grip strength of 80% compared to normal hand by the end of 12 weeks . in ultrasound therapy group 2 protocol , \\xe2\\x81\\x87 out of 27 digits , 13 digits achieved pip + dip'\n",
            " b'although resin composite materials have improved considerably since their introduction , their polymerization shrinkage remains a problem . \\xe2\\x81\\x87 this shrinkage could cause tensile stress and consequent debonding at the tooth - composite interface , which may lead to recurrent caries , postoperative sensitivity , and microleakage . \\xe2\\x81\\x87 several approaches have been proposed to minimize the polymerization shrinkage , such as using an initial low - intensity curing light exposure , incremental placement technique , and applying an intermediate low elastic modulus liner . \\xe2\\x81\\x87 use of low - shrinkage composites is one of the other approaches to control polymerization contraction stress . \\xe2\\x81\\x87 silorane , a new class of ring - opening monomers , is derived from the combination of oxiranes and siloxanes , combining the properties of both , such as hydrophobicity , biocompatibility and low shrinkage . \\xe2\\x81\\x87 previous studies have indicated better enamel and dentin marginal integrity of silorane compared to methacrylate - based composites , while others reported that silorane did not provide better marginal integrity than the methacrylate - based composites . \\xe2\\x81\\x87 other resin composites ( kalore gc , grandio and aelite ls posterior ) used in this study was low shrinkage methacrylate - based composites . \\xe2\\x81\\x87 weakening of the adhesive resin due to cyclic loading is an important issue in restorative dentistry . \\xe2\\x81\\x87 some studies reported increased microleakage of the composite restorations under cyclic loading while others indicated that cyclic loading did not affect the microleakage and marginal integrity of composite restorations . \\xe2\\x81\\x87 scanning electron microscopic ( sem ) evaluation is the gold standard for determination of microleakage in indirect and directly placed adhesive restorations . \\xe2\\x81\\x87 sem - investigation on marginal adaptation of class v cavities might be performed easier because of the smaller size of the cavity and is therefore used more commonly . \\xe2\\x81\\x87 the aim of the current study was to evaluate the effect of cyclic loading on the microleakage of silorane based composite compared with low shrinkage methacrylate - based composites in class v cavities . \\xe2\\x81\\x87 in this experimental study 48 extracted intact human maxillary premolars , without caries , cracks or previous restorations were used . \\xe2\\x81\\x87 the teeth were immersed in 0.5% chloramine t at 4c for 1 week and then stored in physiologic normal saline solution until use . \\xe2\\x81\\x87 class v cavities ( occluso - gingival length of 3 mm , the mesiodistal width of 3 mm , and 1 mm dentinal depth ) were prepared on the buccal and lingual surfaces of the teeth using tapered fissure diamond bur ( tizkavan , tehran , iran ) with water - cooled high - speed handpiece . \\xe2\\x81\\x87 a 0.5 mm , 45c bevel was placed on the enamel margins using a flame - shaped diamond bur ( diatech dental ag ) while gingival margins were prepared at 90c with the external surface . \\xe2\\x81\\x87 the prepared teeth were randomly divided into four groups of 12 teeth each ( 24 cavities ) . \\xe2\\x81\\x87 materials used in this study and their chemical composition in all groups , 37% phosphoric acid gel ( total etch , ivoclar vivadent ) was applied to the enamel part of the cavity for 15 s , rinsed for 15 s and excess water was removed with a light air stream to achieve a moist surface and then restored as follows :  \\xe2\\x81\\x87  group 1 ( siloran system adhesive + filtek p90 ) : the silorane self ecth primer ( 3 m espe , dental product , st paul , usa ) was applied and agitated on dentinal surfaces of cavity for 15 s , gently air - dried , light - cured for 20 s using a led light - curing unit ( guilin woodpecker medical instrument co. , china ) at 900 mw / cm intensity , as checked with a radiometer ( led radiometer demetron , kerr , usa ) after every 10 uses , and the silorane bond was then applied on all surfaces of cavity followed by a gentle stream of air , and cured for 20 s. then each cavity was filled with filtek p90 a3.5 shade composite ( 3 m espe , dental product , st paul , usa ) . in all groups , \\xe2\\x81\\x87 the cavities were filled in three increments : the first increment on the axial wall , the second increment was placed from about the midpoint of the gingival wall to the occlusal cavosurface margin and the third increment filled the remaining of preparation , and each increment was cured for 40 sgroup 2 ( all bond se + aelite ls posterior ) : all bond se ( bisco inc . , schaumburg , usa ) was applied and agitated for 10 s , gently air dried for 5 s and then air dried with greater pressure completely . \\xe2\\x81\\x87 another layer of bonding was applied and the process was repeated again and light cured for 20 s. then each cavity was filled with three layers of aelite ls posterior a3.5 shade composite ( bisco inc . , schaumburg , usa ) and each increment was cured for 40 sgroup 3 ( futurabond nr + grandio ) : a moderately thin layer of futurabond nr ( voco cuxhaven , germany ) was applied for 20 s and air dried for 10 s. another layer of bonding was applied and the process was repeated again and light cured for 20 s. then each cavity was filled with three layers of grandio a3.5 shade composite ( voco cuxhaven , germany ) and each increment was cured for 40 sgroup 4 ( g - bond + kalore - gc ) : g - bond ( gc corporation , tokyo , japan ) was applied and left undisturbed for 10 s. then air dried for 5 s. another layer of g - bond was applied and the process was repeated again and light cured for 20 s. then each cavity was filled with three layers of kalore - gc a3.5 shade composite ( gc corporation , tokyo , japan ) and each increment was cured for 40 s.  \\xe2\\x81\\x87  group 1 ( siloran system adhesive + filtek p90 ) : the silorane self ecth primer ( 3 m espe , dental product , st paul , usa ) was applied and agitated on dentinal surfaces of cavity for 15 s , gently air - dried , light - cured for 20 s using a led light - curing unit ( guilin woodpecker medical instrument co. , china ) at 900 mw / cm intensity , as checked with a radiometer ( led radiometer demetron , kerr , usa ) after every 10 uses , and the silorane bond was then applied on all surfaces of cavity followed by a gentle stream of air , and cured for 20 s. then each cavity was filled with filtek p90 a3.5 shade composite ( 3 m espe , dental product , st paul , usa ) . in all groups , \\xe2\\x81\\x87 the cavities were filled in three increments : the first increment on the axial wall , the second increment was placed from about the midpoint of the gingival wall to the occlusal cavosurface margin and the third increment filled the remaining of preparation , and each increment was cured for 40 s group 2 ( all bond se + aelite ls posterior ) : all bond se ( bisco inc . , schaumburg , usa ) was applied and agitated for 10 s , gently air dried for 5 s and then air dried with greater pressure completely . \\xe2\\x81\\x87 another layer of bonding was applied and the process was repeated again and light cured for 20 s. then each cavity was filled with three layers of aelite ls posterior a3.5 shade composite ( bisco inc . , schaumburg , usa ) and each increment was cured for 40 s group 3 ( futurabond nr + grandio ) : a moderately thin layer of futurabond nr ( voco cuxhaven , germany ) was applied for 20 s and air dried for 10 s. another layer of bonding was applied and the process was repeated again and light cured for 20 s. then each cavity was filled with three layers of grandio a3.5 shade composite ( voco cuxhaven , germany ) and each increment was cured for 40 s group 4 ( g - bond + kalore - gc ) : g - bond ( gc corporation , tokyo , japan ) was applied and left undisturbed for 10 s. then air dried for 5 s. another layer of g - bond was applied and the process was repeated again and light cured for 20 s. then each cavity was filled with three layers of kalore - gc a3.5 shade composite ( gc corporation , tokyo , japan ) and each increment was cured for 40 s. all specimens were finished using fine - grit finishing diamond burs ( diatech dental ag , heerbrug , switzerland ) and polished with sequential disks ( optidisk , kerr , usa ) ( 15 s in each margin ) . \\xe2\\x81\\x87 after storage in an incubator ( malek - teb , iran ) at 37c for 24 h , all teeth were subjected to 2000 thermal cycles of 5c/55c , with a dwell time of 30 s in each bath and a transfer time of 10 s ( malek - teb , iran ) . \\xe2\\x81\\x87 then in each group , half of the teeth ( n = 6 ) were stored in an incubator at 37c and the others were load cycled ( germany , sd mekanotronik ) , as follows : initially , a cylindrical tube was coated with a layer of wax , then the teeth were mounted']\n",
            "\n",
            " Predicted summary:\n",
            " [b'background : zone ii flexor tendon repair after zone ii flexor tendon repair is a challenging procedure .  \\xe2\\x81\\x87  the aim of this study was to evaluate the effectiveness of ultrasound therapy of different frequencies and intensities during the early healing phase of zone ii flexor tendon repair.materials and methods : in this prospective study , we randomly divided 24 patients with zone ii flexor tendon injuries between january 2008 and january 2013 .  \\xe2\\x81\\x87  after surgery , patients were allocated to ultrasound therapy of different frequencies and intensities of ultrasound therapy .  \\xe2\\x81\\x87  ultrasound therapy of 1-mhz frequency at an intensity of 0.7 w / cm2 at an intensity of 0.5 w / cm2 at an intensity of 0.5 w / cm2 at an intensity of 0.5 w / cm2 at an intensity of 0.5 w / cm2 at an intensity of 0.5 w / cm2 at a intensity of 0.5 w / cm2 at a intensity of 0.5 w / cm2 at the intensity of 0.5 w / cm2 was started .  \\xe2\\x81\\x87  ultrasound therapy of 1-mhz frequency at an intensity of 0.5 w / cm2 at an intensity of 0.5 w / cm2 was started from the fifth'\n",
            " b'objective : the aim of this study was to evaluate the effect of cyclic loading on the microleakage of silorane - based composite compared to low shrinkage methacrylate - based composites.materials and methods : in this experimental study 48 extracted human maxillary premolars were randomly divided into four groups ( n = 12 ) .  \\xe2\\x81\\x87  each cavity was filled with the silorane self ecth primer ( voco cuxhaven , germany ) and composite ( kalore gc , grandio and aelite ls posterior ) .  \\xe2\\x81\\x87  then the teeth were subjected to 2000 thermal cycles of 5c/55c and 2000 thermal cycles of 5c/55c with a 30 s dwell time of 30 s. all teeth were subjected to 2000 thermal cycles of 5c/55c and a 30 s dwell time of 10 s. all teeth were subjected to 2000 thermal cycles of 5c/55c and a 30 s dwell time of 10 s. all teeth were subjected to 2000 thermal cycles of 5c/55c and a 30 s dwell time of 10 s. all teeth were subjected to 2000 thermal cycles of 2000 thermal cycles of 5 s/55c and a 30 s dwell time of 10 s .  \\xe2\\x81\\x87 ']\n",
            "\n",
            " Ground truth summary:\n",
            " [b'background : in our institute , most of the patients treated for hand injuries were industrial workers with poor compliance . for rehabilitation after zone ii flexor tendon repair , we had tried various early mobilization protocols . as these protocols demanded a degree of commitment from the patients , our results were suboptimal . hence , to improve the results  \\xe2\\x81\\x87  , we implemented a new rehabilitation protocol by administering the pulsed ultrasound therapy during the early phase of tendon healing.materials and methods : this is a prospective study done over a period of five years from january 2008 to january 2013 .  \\xe2\\x81\\x87  a total of 100 patients and 139 digits with zone ii flexor tendon injuries were studied .  \\xe2\\x81\\x87  after randomization , we administered pulsed ultrasound therapy of different frequencies and intensities for a total of 72 patients and 99 digits and formulated three groups .  \\xe2\\x81\\x87  the results of ultrasound treated cases were compared with each other and with the results of cases treated by immobilization protocol .  \\xe2\\x81\\x87  the results were analyzed using  \\xe2\\x81\\x87  original strickland criteria.results:72% excellent - good results in ultrasound ( group 1 ) protocol , 75% excellent - good results in ultrasound ( group 2 ) protocol , and 77% excellent - good results in ultrasound ( group 3 ) protocol were achieved .  \\xe2\\x81\\x87 '\n",
            " b'background : there are many concerns regarding the marginal seal of composite restorations , especially when composite restorations are subjected to cyclic loading .  \\xe2\\x81\\x87  the aim of this study was to evaluate the effect of cyclic loading on the microleakage of silorane based composite compared with low shrinkage methacrylate - based composites in class v cavities.materials and methods : in this in vitro study , class v cavities were prepared on the facial and lingual surfaces of 48 human premolars ( 96 cavities ) .  \\xe2\\x81\\x87  the teeth were randomly divided into four groups of 12 teeth ( 24 cavities ) each and restored as follows : group 1 ( siloran system adhesive + filtek p90 ) , group 2 ( all bond se + aelite ls posterior ) , group 3 ( futurabond nr + grandio ) , and group 4 ( g - bond + kalore - gc ) .  \\xe2\\x81\\x87  all the specimens were thermocycled for 2000 cycles ( 5 - 55c ) and then half of the specimens from each group , were load cycled .  \\xe2\\x81\\x87  all teeth were immersed in 0.5% basic fuchsine dye , sectioned , and observed under a stereomicr']\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "print('Article:\\n {}\\n\\n Predicted summary:\\n {}\\n\\n Ground truth summary:\\n {}\\n\\n'.format(\n",
        "    tokenizer.detokenize(ex[0]),\n",
        "    tokenizer.detokenize(pred_ids),\n",
        "    tokenizer.detokenize(ex[1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2r55AitoHmmv"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "last_runtime": {},
      "name": "BigBirdGPU.ipynb"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
